{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notebook written by [Zhedong Zheng](https://github.com/zhedongzheng)\n",
    "\n",
    "<img src=\"img/pg.png\" width=\"400\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import gym"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Agent:\n",
    "    def __init__(self, env, n_in, n_out, sess):\n",
    "        self.env = env\n",
    "        self.n_in = n_in\n",
    "        self.n_out = n_out\n",
    "        self.sess = sess\n",
    "        self.build_graph()\n",
    "    # end constructor\n",
    "\n",
    "\n",
    "    def build_graph(self):\n",
    "        self.ph_obs = tf.placeholder(tf.float32, shape=[None, self.n_in])\n",
    "        self.ph_rewards = tf.placeholder(tf.float32, shape=[None])\n",
    "        self.ph_actions = tf.placeholder(tf.int32, shape=[None])\n",
    "        \n",
    "        x = tf.layers.dense(self.ph_obs, 10, tf.tanh)\n",
    "        logits = tf.layers.dense(x, self.n_out)\n",
    "        \n",
    "        self.op_sample = tf.multinomial(tf.nn.log_softmax(logits), num_samples=1)\n",
    "        self.op_action = tf.argmax(logits, -1)\n",
    "\n",
    "        loss = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=self.ph_actions,\n",
    "                                                              logits=logits)\n",
    "        self.op_loss = tf.reduce_mean(loss * self.ph_rewards)\n",
    "        self.op_train = tf.train.AdamOptimizer(2e-3).minimize(self.op_loss)\n",
    "    # end method\n",
    "\n",
    "\n",
    "    def train(self,\n",
    "              nb_episodes=500,\n",
    "              nb_games_per_update=10,\n",
    "              nb_max_steps=1000,\n",
    "              discount_rate=0.99):\n",
    "        \n",
    "        self.sess.run(tf.global_variables_initializer())\n",
    "\n",
    "        for episode in range(nb_episodes):\n",
    "            ep_rewards = []                         # rewards in one eposide\n",
    "            ep_obs = []\n",
    "            ep_actions = []\n",
    "\n",
    "            for game in range(nb_games_per_update):\n",
    "                game_rewards = []                   # rewards in one round of game\n",
    "                obs = self.env.reset()\n",
    "                for step in range(nb_max_steps):\n",
    "                    action_val = self.sess.run(self.op_sample,\n",
    "                                              {self.ph_obs: np.atleast_2d(obs)})\n",
    "                    obs, reward, done, info = self.env.step(action_val[0,0])\n",
    "                    ep_obs.append(obs)\n",
    "                    ep_actions.append(action_val[0,0])\n",
    "                    reward = -5 if done else reward\n",
    "                    game_rewards.append(reward)\n",
    "                    if done:\n",
    "                        break\n",
    "                ep_rewards.append(game_rewards)\n",
    "\n",
    "            ep_rewards = self.discount_and_normalize_rewards(ep_rewards, discount_rate)\n",
    "            flat_rewards = np.concatenate(ep_rewards)\n",
    "            _, loss = self.sess.run([self.op_train, self.op_loss],\n",
    "                                    {self.ph_obs: np.vstack(ep_obs),\n",
    "                                     self.ph_rewards: flat_rewards,\n",
    "                                     self.ph_actions: np.array(ep_actions)})\n",
    "            print(\"Episode %d/%d | Loss: %.3f | Step: %d\" % (episode, nb_episodes, loss, step))\n",
    "    # end method\n",
    "\n",
    "\n",
    "    def simulate(self):\n",
    "        obs = self.env.reset()\n",
    "        done = False\n",
    "        count = 0\n",
    "        while not done:\n",
    "            self.env.render()\n",
    "            action_val = self.sess.run(self.op_action,\n",
    "                                      {self.ph_obs: np.atleast_2d(obs)})\n",
    "            obs, reward, done, info = self.env.step(action_val[0])\n",
    "            count += 1\n",
    "        print(\"Test Time: %d Steps Completed\" % count)\n",
    "    # end method\n",
    "\n",
    "\n",
    "    def discount_rewards(self, game_rewards, discount_rate):\n",
    "        discounted_rewards = np.zeros(len(game_rewards))\n",
    "        cumulative_rewards = 0\n",
    "        for step in reversed(range(len(game_rewards))):\n",
    "            cumulative_rewards = game_rewards[step] + cumulative_rewards * discount_rate\n",
    "            discounted_rewards[step] = cumulative_rewards\n",
    "        return discounted_rewards\n",
    "    # end method\n",
    "\n",
    "\n",
    "    def discount_and_normalize_rewards(self, ep_rewards, discount_rate):\n",
    "        discounted = [self.discount_rewards(game_rewards, discount_rate) for game_rewards in ep_rewards]\n",
    "        flat_rewards = np.concatenate(discounted)\n",
    "        reward_mean = flat_rewards.mean()\n",
    "        reward_std = flat_rewards.std()\n",
    "        return [(game_rewards - reward_mean) / reward_std for game_rewards in discounted]\n",
    "    # end method\n",
    "# end class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main():\n",
    "    agent = Agent(env = gym.make('CartPole-v1'),\n",
    "                  n_in = 4,\n",
    "                  n_out = 2,\n",
    "                  sess = tf.Session())\n",
    "    agent.train()\n",
    "    agent.simulate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n",
      "Episode 0/500 | Loss: 0.055 | Step: 18\n",
      "Episode 1/500 | Loss: 0.080 | Step: 13\n",
      "Episode 2/500 | Loss: 0.110 | Step: 9\n",
      "Episode 3/500 | Loss: 0.049 | Step: 14\n",
      "Episode 4/500 | Loss: 0.040 | Step: 13\n",
      "Episode 5/500 | Loss: 0.026 | Step: 9\n",
      "Episode 6/500 | Loss: 0.065 | Step: 16\n",
      "Episode 7/500 | Loss: -0.052 | Step: 13\n",
      "Episode 8/500 | Loss: 0.053 | Step: 13\n",
      "Episode 9/500 | Loss: 0.034 | Step: 10\n",
      "Episode 10/500 | Loss: 0.062 | Step: 13\n",
      "Episode 11/500 | Loss: 0.087 | Step: 9\n",
      "Episode 12/500 | Loss: 0.058 | Step: 10\n",
      "Episode 13/500 | Loss: 0.040 | Step: 13\n",
      "Episode 14/500 | Loss: 0.052 | Step: 13\n",
      "Episode 15/500 | Loss: 0.023 | Step: 18\n",
      "Episode 16/500 | Loss: -0.070 | Step: 9\n",
      "Episode 17/500 | Loss: 0.005 | Step: 15\n",
      "Episode 18/500 | Loss: 0.011 | Step: 10\n",
      "Episode 19/500 | Loss: 0.003 | Step: 10\n",
      "Episode 20/500 | Loss: 0.032 | Step: 15\n",
      "Episode 21/500 | Loss: 0.057 | Step: 12\n",
      "Episode 22/500 | Loss: 0.030 | Step: 13\n",
      "Episode 23/500 | Loss: -0.007 | Step: 8\n",
      "Episode 24/500 | Loss: 0.032 | Step: 13\n",
      "Episode 25/500 | Loss: 0.057 | Step: 15\n",
      "Episode 26/500 | Loss: 0.014 | Step: 8\n",
      "Episode 27/500 | Loss: 0.026 | Step: 13\n",
      "Episode 28/500 | Loss: 0.002 | Step: 12\n",
      "Episode 29/500 | Loss: 0.024 | Step: 27\n",
      "Episode 30/500 | Loss: 0.013 | Step: 10\n",
      "Episode 31/500 | Loss: 0.049 | Step: 9\n",
      "Episode 32/500 | Loss: 0.026 | Step: 29\n",
      "Episode 33/500 | Loss: 0.009 | Step: 20\n",
      "Episode 34/500 | Loss: 0.021 | Step: 10\n",
      "Episode 35/500 | Loss: 0.025 | Step: 8\n",
      "Episode 36/500 | Loss: 0.035 | Step: 7\n",
      "Episode 37/500 | Loss: 0.031 | Step: 12\n",
      "Episode 38/500 | Loss: 0.005 | Step: 8\n",
      "Episode 39/500 | Loss: 0.026 | Step: 10\n",
      "Episode 40/500 | Loss: -0.007 | Step: 31\n",
      "Episode 41/500 | Loss: 0.028 | Step: 9\n",
      "Episode 42/500 | Loss: 0.031 | Step: 16\n",
      "Episode 43/500 | Loss: 0.048 | Step: 28\n",
      "Episode 44/500 | Loss: -0.041 | Step: 9\n",
      "Episode 45/500 | Loss: 0.012 | Step: 8\n",
      "Episode 46/500 | Loss: -0.003 | Step: 9\n",
      "Episode 47/500 | Loss: 0.031 | Step: 27\n",
      "Episode 48/500 | Loss: 0.034 | Step: 8\n",
      "Episode 49/500 | Loss: -0.018 | Step: 12\n",
      "Episode 50/500 | Loss: 0.027 | Step: 10\n",
      "Episode 51/500 | Loss: 0.004 | Step: 10\n",
      "Episode 52/500 | Loss: 0.016 | Step: 20\n",
      "Episode 53/500 | Loss: 0.019 | Step: 12\n",
      "Episode 54/500 | Loss: 0.002 | Step: 10\n",
      "Episode 55/500 | Loss: 0.015 | Step: 19\n",
      "Episode 56/500 | Loss: 0.010 | Step: 19\n",
      "Episode 57/500 | Loss: -0.032 | Step: 14\n",
      "Episode 58/500 | Loss: 0.012 | Step: 39\n",
      "Episode 59/500 | Loss: -0.007 | Step: 16\n",
      "Episode 60/500 | Loss: -0.000 | Step: 13\n",
      "Episode 61/500 | Loss: 0.027 | Step: 16\n",
      "Episode 62/500 | Loss: 0.006 | Step: 10\n",
      "Episode 63/500 | Loss: -0.013 | Step: 12\n",
      "Episode 64/500 | Loss: 0.004 | Step: 13\n",
      "Episode 65/500 | Loss: -0.004 | Step: 14\n",
      "Episode 66/500 | Loss: 0.022 | Step: 14\n",
      "Episode 67/500 | Loss: 0.016 | Step: 18\n",
      "Episode 68/500 | Loss: -0.006 | Step: 17\n",
      "Episode 69/500 | Loss: 0.005 | Step: 16\n",
      "Episode 70/500 | Loss: -0.006 | Step: 14\n",
      "Episode 71/500 | Loss: -0.013 | Step: 10\n",
      "Episode 72/500 | Loss: -0.007 | Step: 12\n",
      "Episode 73/500 | Loss: 0.000 | Step: 25\n",
      "Episode 74/500 | Loss: -0.019 | Step: 17\n",
      "Episode 75/500 | Loss: 0.003 | Step: 10\n",
      "Episode 76/500 | Loss: -0.004 | Step: 13\n",
      "Episode 77/500 | Loss: -0.008 | Step: 47\n",
      "Episode 78/500 | Loss: -0.019 | Step: 11\n",
      "Episode 79/500 | Loss: -0.028 | Step: 11\n",
      "Episode 80/500 | Loss: -0.008 | Step: 13\n",
      "Episode 81/500 | Loss: -0.001 | Step: 10\n",
      "Episode 82/500 | Loss: -0.006 | Step: 10\n",
      "Episode 83/500 | Loss: -0.009 | Step: 33\n",
      "Episode 84/500 | Loss: -0.009 | Step: 11\n",
      "Episode 85/500 | Loss: -0.007 | Step: 20\n",
      "Episode 86/500 | Loss: -0.004 | Step: 10\n",
      "Episode 87/500 | Loss: -0.016 | Step: 15\n",
      "Episode 88/500 | Loss: -0.015 | Step: 10\n",
      "Episode 89/500 | Loss: -0.022 | Step: 16\n",
      "Episode 90/500 | Loss: -0.026 | Step: 9\n",
      "Episode 91/500 | Loss: -0.017 | Step: 17\n",
      "Episode 92/500 | Loss: -0.022 | Step: 16\n",
      "Episode 93/500 | Loss: -0.011 | Step: 14\n",
      "Episode 94/500 | Loss: -0.019 | Step: 15\n",
      "Episode 95/500 | Loss: -0.022 | Step: 50\n",
      "Episode 96/500 | Loss: -0.016 | Step: 9\n",
      "Episode 97/500 | Loss: -0.015 | Step: 15\n",
      "Episode 98/500 | Loss: -0.021 | Step: 25\n",
      "Episode 99/500 | Loss: -0.021 | Step: 37\n",
      "Episode 100/500 | Loss: -0.024 | Step: 47\n",
      "Episode 101/500 | Loss: -0.041 | Step: 18\n",
      "Episode 102/500 | Loss: -0.034 | Step: 27\n",
      "Episode 103/500 | Loss: -0.020 | Step: 21\n",
      "Episode 104/500 | Loss: -0.038 | Step: 18\n",
      "Episode 105/500 | Loss: -0.021 | Step: 30\n",
      "Episode 106/500 | Loss: -0.026 | Step: 24\n",
      "Episode 107/500 | Loss: -0.018 | Step: 34\n",
      "Episode 108/500 | Loss: -0.036 | Step: 19\n",
      "Episode 109/500 | Loss: -0.024 | Step: 11\n",
      "Episode 110/500 | Loss: -0.030 | Step: 22\n",
      "Episode 111/500 | Loss: -0.035 | Step: 25\n",
      "Episode 112/500 | Loss: -0.020 | Step: 27\n",
      "Episode 113/500 | Loss: -0.014 | Step: 64\n",
      "Episode 114/500 | Loss: -0.019 | Step: 15\n",
      "Episode 115/500 | Loss: -0.032 | Step: 17\n",
      "Episode 116/500 | Loss: -0.030 | Step: 29\n",
      "Episode 117/500 | Loss: -0.021 | Step: 11\n",
      "Episode 118/500 | Loss: -0.019 | Step: 18\n",
      "Episode 119/500 | Loss: -0.019 | Step: 47\n",
      "Episode 120/500 | Loss: -0.027 | Step: 32\n",
      "Episode 121/500 | Loss: -0.012 | Step: 28\n",
      "Episode 122/500 | Loss: -0.046 | Step: 11\n",
      "Episode 123/500 | Loss: -0.033 | Step: 14\n",
      "Episode 124/500 | Loss: -0.011 | Step: 25\n",
      "Episode 125/500 | Loss: -0.029 | Step: 15\n",
      "Episode 126/500 | Loss: -0.016 | Step: 54\n",
      "Episode 127/500 | Loss: -0.043 | Step: 45\n",
      "Episode 128/500 | Loss: -0.025 | Step: 36\n",
      "Episode 129/500 | Loss: -0.041 | Step: 12\n",
      "Episode 130/500 | Loss: -0.046 | Step: 15\n",
      "Episode 131/500 | Loss: -0.020 | Step: 16\n",
      "Episode 132/500 | Loss: -0.021 | Step: 45\n",
      "Episode 133/500 | Loss: -0.030 | Step: 94\n",
      "Episode 134/500 | Loss: -0.015 | Step: 17\n",
      "Episode 135/500 | Loss: -0.016 | Step: 26\n",
      "Episode 136/500 | Loss: -0.016 | Step: 29\n",
      "Episode 137/500 | Loss: -0.014 | Step: 61\n",
      "Episode 138/500 | Loss: -0.032 | Step: 24\n",
      "Episode 139/500 | Loss: -0.001 | Step: 25\n",
      "Episode 140/500 | Loss: -0.008 | Step: 195\n",
      "Episode 141/500 | Loss: -0.027 | Step: 37\n",
      "Episode 142/500 | Loss: -0.017 | Step: 39\n",
      "Episode 143/500 | Loss: -0.016 | Step: 62\n",
      "Episode 144/500 | Loss: -0.036 | Step: 59\n",
      "Episode 145/500 | Loss: -0.018 | Step: 76\n",
      "Episode 146/500 | Loss: -0.030 | Step: 125\n",
      "Episode 147/500 | Loss: -0.015 | Step: 59\n",
      "Episode 148/500 | Loss: -0.008 | Step: 110\n",
      "Episode 149/500 | Loss: -0.015 | Step: 134\n",
      "Episode 150/500 | Loss: -0.027 | Step: 40\n",
      "Episode 151/500 | Loss: -0.021 | Step: 57\n",
      "Episode 152/500 | Loss: -0.018 | Step: 19\n",
      "Episode 153/500 | Loss: -0.028 | Step: 75\n",
      "Episode 154/500 | Loss: -0.020 | Step: 90\n",
      "Episode 155/500 | Loss: -0.021 | Step: 28\n",
      "Episode 156/500 | Loss: -0.017 | Step: 86\n",
      "Episode 157/500 | Loss: -0.003 | Step: 65\n",
      "Episode 158/500 | Loss: -0.022 | Step: 96\n",
      "Episode 159/500 | Loss: -0.013 | Step: 22\n",
      "Episode 160/500 | Loss: -0.003 | Step: 94\n",
      "Episode 161/500 | Loss: -0.037 | Step: 82\n",
      "Episode 162/500 | Loss: -0.016 | Step: 135\n",
      "Episode 163/500 | Loss: -0.016 | Step: 11\n",
      "Episode 164/500 | Loss: -0.036 | Step: 88\n",
      "Episode 165/500 | Loss: -0.046 | Step: 13\n",
      "Episode 166/500 | Loss: -0.026 | Step: 13\n",
      "Episode 167/500 | Loss: -0.025 | Step: 22\n",
      "Episode 168/500 | Loss: -0.013 | Step: 30\n",
      "Episode 169/500 | Loss: -0.021 | Step: 128\n",
      "Episode 170/500 | Loss: -0.010 | Step: 71\n",
      "Episode 171/500 | Loss: -0.009 | Step: 151\n",
      "Episode 172/500 | Loss: -0.020 | Step: 61\n",
      "Episode 173/500 | Loss: -0.004 | Step: 128\n",
      "Episode 174/500 | Loss: -0.011 | Step: 179\n",
      "Episode 175/500 | Loss: -0.022 | Step: 138\n",
      "Episode 176/500 | Loss: -0.018 | Step: 61\n",
      "Episode 177/500 | Loss: -0.012 | Step: 14\n",
      "Episode 178/500 | Loss: -0.007 | Step: 53\n",
      "Episode 179/500 | Loss: -0.014 | Step: 130\n",
      "Episode 180/500 | Loss: -0.007 | Step: 155\n",
      "Episode 181/500 | Loss: -0.014 | Step: 49\n",
      "Episode 182/500 | Loss: -0.018 | Step: 80\n",
      "Episode 183/500 | Loss: -0.013 | Step: 28\n",
      "Episode 184/500 | Loss: -0.010 | Step: 99\n",
      "Episode 185/500 | Loss: -0.010 | Step: 153\n",
      "Episode 186/500 | Loss: -0.025 | Step: 231\n",
      "Episode 187/500 | Loss: -0.023 | Step: 111\n",
      "Episode 188/500 | Loss: -0.021 | Step: 191\n",
      "Episode 189/500 | Loss: -0.020 | Step: 186\n",
      "Episode 190/500 | Loss: -0.019 | Step: 267\n",
      "Episode 191/500 | Loss: -0.036 | Step: 66\n",
      "Episode 192/500 | Loss: -0.020 | Step: 224\n",
      "Episode 193/500 | Loss: -0.026 | Step: 56\n",
      "Episode 194/500 | Loss: -0.020 | Step: 78\n",
      "Episode 195/500 | Loss: -0.017 | Step: 213\n",
      "Episode 196/500 | Loss: -0.012 | Step: 200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 197/500 | Loss: -0.009 | Step: 33\n",
      "Episode 198/500 | Loss: -0.007 | Step: 190\n",
      "Episode 199/500 | Loss: -0.011 | Step: 319\n",
      "Episode 200/500 | Loss: -0.023 | Step: 102\n",
      "Episode 201/500 | Loss: -0.014 | Step: 190\n",
      "Episode 202/500 | Loss: -0.020 | Step: 211\n",
      "Episode 203/500 | Loss: -0.021 | Step: 187\n",
      "Episode 204/500 | Loss: -0.012 | Step: 305\n",
      "Episode 205/500 | Loss: -0.019 | Step: 170\n",
      "Episode 206/500 | Loss: -0.021 | Step: 134\n",
      "Episode 207/500 | Loss: -0.025 | Step: 102\n",
      "Episode 208/500 | Loss: -0.028 | Step: 53\n",
      "Episode 209/500 | Loss: -0.022 | Step: 56\n",
      "Episode 210/500 | Loss: -0.022 | Step: 60\n",
      "Episode 211/500 | Loss: -0.016 | Step: 123\n",
      "Episode 212/500 | Loss: -0.019 | Step: 213\n",
      "Episode 213/500 | Loss: -0.013 | Step: 121\n",
      "Episode 214/500 | Loss: -0.018 | Step: 202\n",
      "Episode 215/500 | Loss: -0.018 | Step: 245\n",
      "Episode 216/500 | Loss: -0.020 | Step: 109\n",
      "Episode 217/500 | Loss: -0.016 | Step: 106\n",
      "Episode 218/500 | Loss: -0.028 | Step: 138\n",
      "Episode 219/500 | Loss: -0.024 | Step: 40\n",
      "Episode 220/500 | Loss: -0.016 | Step: 146\n",
      "Episode 221/500 | Loss: -0.015 | Step: 43\n",
      "Episode 222/500 | Loss: -0.024 | Step: 103\n",
      "Episode 223/500 | Loss: -0.018 | Step: 216\n",
      "Episode 224/500 | Loss: -0.009 | Step: 134\n",
      "Episode 225/500 | Loss: -0.008 | Step: 186\n",
      "Episode 226/500 | Loss: -0.010 | Step: 274\n",
      "Episode 227/500 | Loss: -0.023 | Step: 72\n",
      "Episode 228/500 | Loss: -0.023 | Step: 110\n",
      "Episode 229/500 | Loss: -0.026 | Step: 138\n",
      "Episode 230/500 | Loss: -0.027 | Step: 15\n",
      "Episode 231/500 | Loss: -0.020 | Step: 119\n",
      "Episode 232/500 | Loss: -0.014 | Step: 114\n",
      "Episode 233/500 | Loss: -0.028 | Step: 137\n",
      "Episode 234/500 | Loss: -0.022 | Step: 13\n",
      "Episode 235/500 | Loss: -0.017 | Step: 403\n",
      "Episode 236/500 | Loss: -0.023 | Step: 336\n",
      "Episode 237/500 | Loss: -0.017 | Step: 195\n",
      "Episode 238/500 | Loss: -0.030 | Step: 216\n",
      "Episode 239/500 | Loss: -0.010 | Step: 286\n",
      "Episode 240/500 | Loss: -0.013 | Step: 161\n",
      "Episode 241/500 | Loss: -0.020 | Step: 96\n",
      "Episode 242/500 | Loss: -0.010 | Step: 120\n",
      "Episode 243/500 | Loss: -0.020 | Step: 131\n",
      "Episode 244/500 | Loss: -0.016 | Step: 44\n",
      "Episode 245/500 | Loss: -0.024 | Step: 328\n",
      "Episode 246/500 | Loss: -0.020 | Step: 207\n",
      "Episode 247/500 | Loss: -0.021 | Step: 216\n",
      "Episode 248/500 | Loss: -0.019 | Step: 201\n",
      "Episode 249/500 | Loss: -0.014 | Step: 125\n",
      "Episode 250/500 | Loss: -0.039 | Step: 72\n",
      "Episode 251/500 | Loss: -0.035 | Step: 138\n",
      "Episode 252/500 | Loss: -0.023 | Step: 80\n",
      "Episode 253/500 | Loss: -0.022 | Step: 317\n",
      "Episode 254/500 | Loss: -0.017 | Step: 89\n",
      "Episode 255/500 | Loss: -0.021 | Step: 50\n",
      "Episode 256/500 | Loss: -0.022 | Step: 108\n",
      "Episode 257/500 | Loss: -0.008 | Step: 269\n",
      "Episode 258/500 | Loss: -0.037 | Step: 114\n",
      "Episode 259/500 | Loss: -0.018 | Step: 237\n",
      "Episode 260/500 | Loss: -0.012 | Step: 499\n",
      "Episode 261/500 | Loss: -0.013 | Step: 183\n",
      "Episode 262/500 | Loss: -0.028 | Step: 360\n",
      "Episode 263/500 | Loss: -0.039 | Step: 21\n",
      "Episode 264/500 | Loss: -0.025 | Step: 38\n",
      "Episode 265/500 | Loss: -0.015 | Step: 487\n",
      "Episode 266/500 | Loss: -0.026 | Step: 227\n",
      "Episode 267/500 | Loss: -0.011 | Step: 357\n",
      "Episode 268/500 | Loss: -0.023 | Step: 408\n",
      "Episode 269/500 | Loss: -0.012 | Step: 451\n",
      "Episode 270/500 | Loss: -0.025 | Step: 499\n",
      "Episode 271/500 | Loss: -0.035 | Step: 429\n",
      "Episode 272/500 | Loss: -0.025 | Step: 388\n",
      "Episode 273/500 | Loss: -0.022 | Step: 201\n",
      "Episode 274/500 | Loss: -0.030 | Step: 289\n",
      "Episode 275/500 | Loss: -0.029 | Step: 179\n",
      "Episode 276/500 | Loss: -0.030 | Step: 52\n",
      "Episode 277/500 | Loss: -0.025 | Step: 289\n",
      "Episode 278/500 | Loss: -0.017 | Step: 256\n",
      "Episode 279/500 | Loss: -0.019 | Step: 405\n",
      "Episode 280/500 | Loss: -0.016 | Step: 246\n",
      "Episode 281/500 | Loss: -0.019 | Step: 142\n",
      "Episode 282/500 | Loss: -0.035 | Step: 427\n",
      "Episode 283/500 | Loss: -0.025 | Step: 173\n",
      "Episode 284/500 | Loss: -0.017 | Step: 499\n",
      "Episode 285/500 | Loss: -0.020 | Step: 499\n",
      "Episode 286/500 | Loss: -0.018 | Step: 274\n",
      "Episode 287/500 | Loss: -0.031 | Step: 499\n",
      "Episode 288/500 | Loss: -0.018 | Step: 317\n",
      "Episode 289/500 | Loss: -0.026 | Step: 276\n",
      "Episode 290/500 | Loss: -0.024 | Step: 295\n",
      "Episode 291/500 | Loss: -0.021 | Step: 209\n",
      "Episode 292/500 | Loss: -0.035 | Step: 447\n",
      "Episode 293/500 | Loss: -0.030 | Step: 396\n",
      "Episode 294/500 | Loss: -0.034 | Step: 499\n",
      "Episode 295/500 | Loss: -0.018 | Step: 69\n",
      "Episode 296/500 | Loss: -0.031 | Step: 114\n",
      "Episode 297/500 | Loss: -0.010 | Step: 499\n",
      "Episode 298/500 | Loss: -0.041 | Step: 288\n",
      "Episode 299/500 | Loss: -0.022 | Step: 161\n",
      "Episode 300/500 | Loss: -0.014 | Step: 258\n",
      "Episode 301/500 | Loss: -0.020 | Step: 387\n",
      "Episode 302/500 | Loss: -0.027 | Step: 327\n",
      "Episode 303/500 | Loss: -0.026 | Step: 161\n",
      "Episode 304/500 | Loss: -0.016 | Step: 72\n",
      "Episode 305/500 | Loss: -0.021 | Step: 400\n",
      "Episode 306/500 | Loss: -0.025 | Step: 67\n",
      "Episode 307/500 | Loss: -0.015 | Step: 499\n",
      "Episode 308/500 | Loss: -0.032 | Step: 153\n",
      "Episode 309/500 | Loss: -0.013 | Step: 165\n",
      "Episode 310/500 | Loss: -0.015 | Step: 289\n",
      "Episode 311/500 | Loss: -0.020 | Step: 147\n",
      "Episode 312/500 | Loss: -0.016 | Step: 276\n",
      "Episode 313/500 | Loss: -0.015 | Step: 19\n",
      "Episode 314/500 | Loss: -0.019 | Step: 102\n",
      "Episode 315/500 | Loss: -0.024 | Step: 219\n",
      "Episode 316/500 | Loss: -0.016 | Step: 499\n",
      "Episode 317/500 | Loss: -0.029 | Step: 132\n",
      "Episode 318/500 | Loss: -0.014 | Step: 499\n",
      "Episode 319/500 | Loss: -0.010 | Step: 256\n",
      "Episode 320/500 | Loss: -0.017 | Step: 290\n",
      "Episode 321/500 | Loss: -0.010 | Step: 207\n",
      "Episode 322/500 | Loss: -0.018 | Step: 191\n",
      "Episode 323/500 | Loss: -0.016 | Step: 499\n",
      "Episode 324/500 | Loss: -0.016 | Step: 124\n",
      "Episode 325/500 | Loss: -0.030 | Step: 171\n",
      "Episode 326/500 | Loss: -0.025 | Step: 96\n",
      "Episode 327/500 | Loss: -0.022 | Step: 317\n",
      "Episode 328/500 | Loss: -0.025 | Step: 430\n",
      "Episode 329/500 | Loss: -0.023 | Step: 192\n",
      "Episode 330/500 | Loss: -0.011 | Step: 203\n",
      "Episode 331/500 | Loss: -0.023 | Step: 447\n",
      "Episode 332/500 | Loss: -0.017 | Step: 253\n",
      "Episode 333/500 | Loss: -0.025 | Step: 499\n",
      "Episode 334/500 | Loss: -0.020 | Step: 499\n",
      "Episode 335/500 | Loss: -0.020 | Step: 358\n",
      "Episode 336/500 | Loss: -0.018 | Step: 306\n",
      "Episode 337/500 | Loss: -0.018 | Step: 264\n",
      "Episode 338/500 | Loss: -0.022 | Step: 178\n",
      "Episode 339/500 | Loss: -0.024 | Step: 499\n",
      "Episode 340/500 | Loss: -0.026 | Step: 491\n",
      "Episode 341/500 | Loss: -0.012 | Step: 268\n",
      "Episode 342/500 | Loss: -0.017 | Step: 499\n",
      "Episode 343/500 | Loss: -0.025 | Step: 182\n",
      "Episode 344/500 | Loss: -0.011 | Step: 499\n",
      "Episode 345/500 | Loss: -0.018 | Step: 499\n",
      "Episode 346/500 | Loss: -0.026 | Step: 344\n",
      "Episode 347/500 | Loss: -0.007 | Step: 499\n",
      "Episode 348/500 | Loss: -0.021 | Step: 428\n",
      "Episode 349/500 | Loss: -0.019 | Step: 191\n",
      "Episode 350/500 | Loss: -0.012 | Step: 499\n",
      "Episode 351/500 | Loss: -0.022 | Step: 499\n",
      "Episode 352/500 | Loss: -0.030 | Step: 460\n",
      "Episode 353/500 | Loss: -0.018 | Step: 472\n",
      "Episode 354/500 | Loss: -0.020 | Step: 499\n",
      "Episode 355/500 | Loss: -0.014 | Step: 499\n",
      "Episode 356/500 | Loss: -0.029 | Step: 499\n",
      "Episode 357/500 | Loss: -0.033 | Step: 312\n",
      "Episode 358/500 | Loss: -0.018 | Step: 499\n",
      "Episode 359/500 | Loss: -0.019 | Step: 499\n",
      "Episode 360/500 | Loss: -0.011 | Step: 499\n",
      "Episode 361/500 | Loss: -0.019 | Step: 499\n",
      "Episode 362/500 | Loss: -0.028 | Step: 376\n",
      "Episode 363/500 | Loss: -0.014 | Step: 499\n",
      "Episode 364/500 | Loss: -0.011 | Step: 191\n",
      "Episode 365/500 | Loss: -0.011 | Step: 219\n",
      "Episode 366/500 | Loss: -0.031 | Step: 361\n",
      "Episode 367/500 | Loss: -0.014 | Step: 210\n",
      "Episode 368/500 | Loss: -0.027 | Step: 477\n",
      "Episode 369/500 | Loss: -0.015 | Step: 74\n",
      "Episode 370/500 | Loss: -0.011 | Step: 499\n",
      "Episode 371/500 | Loss: -0.019 | Step: 179\n",
      "Episode 372/500 | Loss: -0.019 | Step: 259\n",
      "Episode 373/500 | Loss: -0.006 | Step: 499\n",
      "Episode 374/500 | Loss: -0.017 | Step: 499\n",
      "Episode 375/500 | Loss: -0.029 | Step: 499\n",
      "Episode 376/500 | Loss: -0.018 | Step: 420\n",
      "Episode 377/500 | Loss: -0.004 | Step: 499\n",
      "Episode 378/500 | Loss: -0.006 | Step: 400\n",
      "Episode 379/500 | Loss: -0.010 | Step: 182\n",
      "Episode 380/500 | Loss: -0.019 | Step: 499\n",
      "Episode 381/500 | Loss: -0.018 | Step: 122\n",
      "Episode 382/500 | Loss: -0.011 | Step: 450\n",
      "Episode 383/500 | Loss: -0.022 | Step: 499\n",
      "Episode 384/500 | Loss: -0.013 | Step: 433\n",
      "Episode 385/500 | Loss: -0.021 | Step: 267\n",
      "Episode 386/500 | Loss: -0.007 | Step: 499\n",
      "Episode 387/500 | Loss: -0.011 | Step: 499\n",
      "Episode 388/500 | Loss: -0.015 | Step: 499\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episode 389/500 | Loss: -0.019 | Step: 499\n",
      "Episode 390/500 | Loss: -0.027 | Step: 499\n",
      "Episode 391/500 | Loss: -0.014 | Step: 234\n",
      "Episode 392/500 | Loss: -0.029 | Step: 401\n",
      "Episode 393/500 | Loss: -0.014 | Step: 499\n",
      "Episode 394/500 | Loss: -0.014 | Step: 396\n",
      "Episode 395/500 | Loss: -0.022 | Step: 499\n",
      "Episode 396/500 | Loss: -0.018 | Step: 418\n",
      "Episode 397/500 | Loss: -0.024 | Step: 82\n",
      "Episode 398/500 | Loss: -0.019 | Step: 243\n",
      "Episode 399/500 | Loss: -0.034 | Step: 288\n",
      "Episode 400/500 | Loss: -0.031 | Step: 499\n",
      "Episode 401/500 | Loss: -0.007 | Step: 82\n",
      "Episode 402/500 | Loss: -0.011 | Step: 499\n",
      "Episode 403/500 | Loss: -0.016 | Step: 143\n",
      "Episode 404/500 | Loss: -0.028 | Step: 444\n",
      "Episode 405/500 | Loss: -0.012 | Step: 499\n",
      "Episode 406/500 | Loss: -0.017 | Step: 499\n",
      "Episode 407/500 | Loss: -0.016 | Step: 227\n",
      "Episode 408/500 | Loss: -0.008 | Step: 24\n",
      "Episode 409/500 | Loss: -0.010 | Step: 499\n",
      "Episode 410/500 | Loss: -0.011 | Step: 499\n",
      "Episode 411/500 | Loss: -0.022 | Step: 277\n",
      "Episode 412/500 | Loss: -0.013 | Step: 499\n",
      "Episode 413/500 | Loss: 0.000 | Step: 499\n",
      "Episode 414/500 | Loss: -0.012 | Step: 499\n",
      "Episode 415/500 | Loss: -0.005 | Step: 499\n",
      "Episode 416/500 | Loss: -0.011 | Step: 127\n",
      "Episode 417/500 | Loss: -0.031 | Step: 499\n",
      "Episode 418/500 | Loss: -0.009 | Step: 499\n",
      "Episode 419/500 | Loss: -0.020 | Step: 208\n",
      "Episode 420/500 | Loss: -0.020 | Step: 499\n",
      "Episode 421/500 | Loss: 0.001 | Step: 499\n",
      "Episode 422/500 | Loss: -0.016 | Step: 325\n",
      "Episode 423/500 | Loss: -0.001 | Step: 152\n",
      "Episode 424/500 | Loss: -0.014 | Step: 499\n",
      "Episode 425/500 | Loss: -0.005 | Step: 499\n",
      "Episode 426/500 | Loss: -0.003 | Step: 499\n",
      "Episode 427/500 | Loss: -0.020 | Step: 499\n",
      "Episode 428/500 | Loss: 0.005 | Step: 499\n",
      "Episode 429/500 | Loss: -0.013 | Step: 499\n",
      "Episode 430/500 | Loss: -0.009 | Step: 499\n",
      "Episode 431/500 | Loss: -0.006 | Step: 489\n",
      "Episode 432/500 | Loss: -0.018 | Step: 499\n",
      "Episode 433/500 | Loss: -0.009 | Step: 499\n",
      "Episode 434/500 | Loss: -0.015 | Step: 201\n",
      "Episode 435/500 | Loss: -0.009 | Step: 499\n",
      "Episode 436/500 | Loss: -0.004 | Step: 499\n",
      "Episode 437/500 | Loss: -0.003 | Step: 499\n",
      "Episode 438/500 | Loss: -0.007 | Step: 499\n",
      "Episode 439/500 | Loss: -0.003 | Step: 499\n",
      "Episode 440/500 | Loss: -0.011 | Step: 499\n",
      "Episode 441/500 | Loss: -0.020 | Step: 499\n",
      "Episode 442/500 | Loss: -0.005 | Step: 499\n",
      "Episode 443/500 | Loss: -0.019 | Step: 499\n",
      "Episode 444/500 | Loss: -0.013 | Step: 499\n",
      "Episode 445/500 | Loss: -0.002 | Step: 499\n",
      "Episode 446/500 | Loss: -0.010 | Step: 499\n",
      "Episode 447/500 | Loss: -0.008 | Step: 327\n",
      "Episode 448/500 | Loss: -0.010 | Step: 499\n",
      "Episode 449/500 | Loss: -0.008 | Step: 279\n",
      "Episode 450/500 | Loss: -0.010 | Step: 419\n",
      "Episode 451/500 | Loss: -0.014 | Step: 243\n",
      "Episode 452/500 | Loss: -0.002 | Step: 499\n",
      "Episode 453/500 | Loss: -0.012 | Step: 499\n",
      "Episode 454/500 | Loss: -0.011 | Step: 499\n",
      "Episode 455/500 | Loss: -0.006 | Step: 124\n",
      "Episode 456/500 | Loss: -0.011 | Step: 499\n",
      "Episode 457/500 | Loss: -0.026 | Step: 499\n",
      "Episode 458/500 | Loss: -0.021 | Step: 221\n",
      "Episode 459/500 | Loss: -0.017 | Step: 499\n",
      "Episode 460/500 | Loss: -0.013 | Step: 499\n",
      "Episode 461/500 | Loss: -0.012 | Step: 499\n",
      "Episode 462/500 | Loss: -0.005 | Step: 433\n",
      "Episode 463/500 | Loss: -0.019 | Step: 499\n",
      "Episode 464/500 | Loss: -0.012 | Step: 499\n",
      "Episode 465/500 | Loss: -0.014 | Step: 174\n",
      "Episode 466/500 | Loss: -0.005 | Step: 474\n",
      "Episode 467/500 | Loss: -0.016 | Step: 499\n",
      "Episode 468/500 | Loss: -0.020 | Step: 499\n",
      "Episode 469/500 | Loss: -0.019 | Step: 363\n",
      "Episode 470/500 | Loss: -0.006 | Step: 499\n",
      "Episode 471/500 | Loss: -0.002 | Step: 499\n",
      "Episode 472/500 | Loss: 0.004 | Step: 499\n",
      "Episode 473/500 | Loss: -0.001 | Step: 485\n",
      "Episode 474/500 | Loss: -0.002 | Step: 499\n",
      "Episode 475/500 | Loss: -0.011 | Step: 499\n",
      "Episode 476/500 | Loss: -0.014 | Step: 499\n",
      "Episode 477/500 | Loss: -0.021 | Step: 499\n",
      "Episode 478/500 | Loss: -0.009 | Step: 185\n",
      "Episode 479/500 | Loss: -0.024 | Step: 211\n",
      "Episode 480/500 | Loss: -0.009 | Step: 499\n",
      "Episode 481/500 | Loss: 0.001 | Step: 499\n",
      "Episode 482/500 | Loss: -0.008 | Step: 499\n",
      "Episode 483/500 | Loss: -0.007 | Step: 499\n",
      "Episode 484/500 | Loss: -0.007 | Step: 499\n",
      "Episode 485/500 | Loss: -0.018 | Step: 499\n",
      "Episode 486/500 | Loss: -0.010 | Step: 499\n",
      "Episode 487/500 | Loss: -0.004 | Step: 499\n",
      "Episode 488/500 | Loss: -0.002 | Step: 499\n",
      "Episode 489/500 | Loss: -0.012 | Step: 499\n",
      "Episode 490/500 | Loss: -0.011 | Step: 499\n",
      "Episode 491/500 | Loss: -0.009 | Step: 499\n",
      "Episode 492/500 | Loss: -0.016 | Step: 499\n",
      "Episode 493/500 | Loss: -0.013 | Step: 205\n",
      "Episode 494/500 | Loss: -0.027 | Step: 499\n",
      "Episode 495/500 | Loss: 0.000 | Step: 499\n",
      "Episode 496/500 | Loss: -0.005 | Step: 499\n",
      "Episode 497/500 | Loss: -0.013 | Step: 499\n",
      "Episode 498/500 | Loss: 0.006 | Step: 499\n",
      "Episode 499/500 | Loss: -0.022 | Step: 499\n",
      "Test Time: 500 Steps Completed\n"
     ]
    }
   ],
   "source": [
    "main()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
